package com.carl.main;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.carl.mapper.KrMapper;
import com.carl.model.Kr;
import com.carl.utils.MybatisHelper;
import org.apache.ibatis.session.SqlSession;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.Date;

public class Day03_36KrCrawler {

    /**
        regex - hot : div.hotlist-main > div
        regex - jsonData : body > script:nth-child(2)
        regex - krData :
                    href : a.hotlist-item-toptwo-title
                    title : krData[title]
                    upTime : krData[published_at]
     **/
    // 36kr : https://36kr.com
    public static void main(String[] args)throws IOException {
         SqlSession sqlSession = MybatisHelper.getSqlSessionLocal();
         KrMapper krMapper = sqlSession.getMapper(KrMapper.class);
         String crawlerUrl = "https://36kr.com";
         Document document = Jsoup.connect(crawlerUrl).get();

         Elements elements = document.select("div.hotlist-main > div");
         String krDataText = document.select("body > script:nth-child(2)").
                                    html().substring("window.initialState=".length());
         JSONArray jsonArray = JSON.parseObject(krDataText).getJSONObject("homeData").
                                getJSONObject("data").getJSONObject("hotlist").getJSONArray("data");
         for (int i = 0; i < elements.size(); i++) {
             Kr kr = new Kr();
             Element element = elements.get(i);
             String href = element.select("a.hotlist-item-toptwo-title").attr("href");
             if (href.equals("") || href == null) {
                 href = element.select("a.hotlist-item-other-title").attr("href");
             }
             String title = jsonArray.getJSONObject(i).getString("title");
             String upTime = jsonArray.getJSONObject(i).getString("published_at");

             kr.setTitle(title);
             kr.setUpTime(upTime);
             kr.setUrl(crawlerUrl + href);
             kr.setCrawlerTime(new Date());
             krMapper.insert(kr);
         }
         sqlSession.commit();
         sqlSession.close();
     }

}
